package com.xiaotu.spider.maoyan.pageprocessor;

import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;

import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Component;

import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.model.MovieCityModel;
import com.xiaotu.common.model.MovieMarketModel;
import com.xiaotu.common.model.MoviePeopleModel;
import com.xiaotu.common.model.MovieRecordModel;
import com.xiaotu.common.model.MovieShadowModel;
import com.xiaotu.common.model.MovieSplitModel;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.DateUtils;
import com.xiaotu.common.util.GsonUtils;
import com.xiaotu.common.util.HtmlUtil;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.PropertiesUtil;
import com.xiaotu.common.util.RegexUtils;
import com.xiaotu.common.util.SepratorUtil;
import com.xiaotu.common.util.StringUtil;
import com.xiaotu.spider.AUrlPageProcessor;

import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;

/**
 * @类名 MaoYanPageProcessor
 * @日期 2017年7月31日
 * @作者 高海军
 * @功能 猫眼票房爬取
 */
@Component("MaoYanPageProcessor")
public class MaoYanPageProcessor extends AUrlPageProcessor
{

	private Map<String,String> cookies = new HashMap<String,String>();
	
	{
		cookies.put("Cookie", PropertiesUtil.getProperty("BoxOffice.site.Cookie"));
	}
	
    public static final String LIST_URL = "http://piaofang.maoyan.com/dashboard-ajax/movie?showDate=%s&orderType=0";//列表地址

    public static final String CITY_URL = "http://piaofang.maoyan.com/movie/%s/box-ajax?boxLevel=4&date=%s&offset=0&type=areaBox&method=change";//分城票房

    public static final String SHADOW_URL = "http://piaofang.maoyan.com/movie/%s/box-ajax?boxLevel=5&date=%s&offset=0&type=shadowBox&method=change";//影投票房

    public static final String PEOPLE_URL = "https://piaofang.maoyan.com/movie/%s/wantindex?wkwebview=1&city_tier=0&city_id=0&cityName=全国";//受众票房

    public static final String DETAIL_URL = "http://piaofang.maoyan.com/movie/%s?_v_=yes";//详细页面

    public static final String MARKET_URL = "http://piaofang.maoyan.com/movie/%s/marketing";//营销事件

    private String beginDate = DateUtils.getDateString(new Date(), DateUtils
            .FILE_NOMENCLATURE_FORMAT);
    
    private String nowDate = DateUtils.getDateString(new Date(), DateUtils
            .DAY_FORMAT);
    

    @Override
    public void initRunParams(Map<String, Object> map)
    {
        if (map.get("beginDate") != null)
            this.beginDate = map.get("beginDate").toString();
    }

    @SuppressWarnings("unchecked")
    @Override
    public void process(Page page)
    {
        switch (this.getPageDataType(page))
        {
            case Constants.PageDataType.TV_RECORD:
                this.detailProcess(page);
                break;
            case Constants.PageDataType.TV_CITY:
                this.cityProcess(page);
                break;
            case Constants.PageDataType.MOVIE_COMPANY:
                this.companyProcess(page);
                break;
            case Constants.PageDataType.TV_AUDIENCE:
                this.peopleProcess(page);
                break;
            case Constants.PageDataType.MOVIE_MARKET:
                this.marketProcess(page);
                break;
            default:
                this.listProcess(page);
                break;
        }
    }

    @SuppressWarnings("unchecked")
	private void listProcess(Page page)
    {
        String json = page.getRawText();
        if (StringUtils.isEmpty(json))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        Map<String, Object> map = GsonUtils.fromJson(json, Map.class);
        if (map == null || map.isEmpty() || map.get("movieList") == null)
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        Map<String, Object> dataMap = ((Map<String, Object>) map.get("movieList"));
        List<Map<String, Object>> movList = (List<Map<String, Object>>) dataMap.get("list");

        if (movList == null || movList.isEmpty())
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        List<MovieRecordModel> list = new ArrayList<>();
        movList.forEach(movMap ->
        {

            Map<String, Object> movieInfo = (Map<String, Object>) movMap.get("movieInfo");
            String dataId = RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, StringUtil
                    .nullToString(movieInfo.get("movieId")));
            Map<String, Object> boxSplitUnit = (Map<String, Object>) movMap.get("boxSplitUnit");
            String box = StringUtil.nullToString(boxSplitUnit.get("num"))+ StringUtil.nullToString(boxSplitUnit.get("unit"));

            String releaseDateCount = RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, StringUtil
                    .nullToString(movieInfo.get("releaseInfo")));

            String queryDate = DateUtils.getBeforeDayDynamic(new Date(),1);

            list.add(new MovieRecordModel(Constants.TargetType.OTHER_TYPE_ID,
                    StringUtil.nullToString(movieInfo.get("movieName")),
                    StringUtil.nullToString(box), // 当日票房
                    StringUtil.nullToString(movMap.get("sumBoxDesc")), // 总票房
                    StringUtil.nullToString(movMap.get("showCountRate")), // 排片比
                    StringUtil.nullToString(movMap.get("boxRate")), // 票房百分比
                    StringUtil.nullToString(movMap.get("showCount")), // 当日场次
                    StringUtil.nullToString(releaseDateCount), // 上映天数
                    StringUtil.nullToString(queryDate), // 上映日期
                    StringUtil.nullToString(movMap.get("avgSeatView")), // 上座率
                    StringUtil.nullToString(movMap.get("avgShowView")), // 场均人次
                    null, // 座位数
                    StringUtil.nullToString(null), // 排座占比
                    Constants.SiteName.MAOYAN, //网站名称
                    dataId)//网站电影id
            );
            if (StringUtils.isEmpty(dataId))
                return;
            Stream.of(SHADOW_URL, CITY_URL, DETAIL_URL, PEOPLE_URL, MARKET_URL).forEach(url ->
                    PageUtils.addTargetRequest(page, String.format(url, dataId , nowDate), page.getRequest
                            ().getExtras()));
        });

        page.putField(MovieRecordModel.class.getName(), list);
    }

    private void marketProcess(Page page)
    {
        List<String> divList = page.getHtml().xpath("//div[@class='trace-main']").all();
        if (divList == null || divList.isEmpty())
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        List<MovieMarketModel> marketList = new ArrayList<>(divList.size());
        String dataId = RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, page.getUrl().toString());
        divList.forEach(divStr ->
        {
            Html html = new Html(divStr);
            marketList.add(new MovieMarketModel(dataId, Constants.SiteName.MAOYAN, html.xpath
                    ("//span[@class='trace-name flex-item']/text()").get(), html.xpath
                    ("//span[@class='date-str']/text()").get(), html.xpath
                    ("//p[@class='trace-content']/text()").get(), html.xpath
                    ("//span[@class='trace-tip flex-item']/em/text()").get(), html.xpath
                    ("//span[@class='date-desc']/text()").get()));
        });
        page.putField(MovieMarketModel.class.getName(), marketList);
    }

    private void detailProcess(Page page)
    {
    	//解密加密数据
		Html html = PageUtils.spiderAudienceRating(page,cookies);
    	
        String boxOffice =html.xpath
        		("//div[@class='movie-container']//div[@class='info-detail-row']").all().get(1);
        Html htmls = new Html(boxOffice);
        List<String> all = htmls.xpath("//div[@class='info-detail-row']//div").all();
        if (all == null || all.size() < 8 || StringUtils.isEmpty(boxOffice))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        page.putField(MovieSplitModel.class.getName(), new MovieSplitModel(RegexUtils.regexSearch
                (RegexUtils.REGEX_INTEGER, page.getUrl().toString()), Constants.SiteName.MAOYAN,
                getFormattedData(new Html(all.get(0)).xpath("//p[@class='info-detail-content']").get()), getFormattedData(new Html(all.get(3)).xpath("//p[@class='info-detail-content']").get()), 
                getFormattedData(new Html(all.get(5)).xpath("//p[@class='info-detail-content']").get()),getFormattedData(new Html(all.get(7)).xpath("//p[@class='info-detail-content']").get())));
    }

    private String getFormattedData(String data)
    {
        return RegexUtils.replaceStr(HtmlUtil.delHTMLTag(data), SepratorUtil.SEP_SPACE_EN,
                StringUtils.EMPTY);
    }

    /**
     * 分城数据爬取
     *
     * @param page
     */
    private void cityProcess(Page page)
    {
    	String rawText = page.getRawText();
		if(null == rawText || StringUtils.isBlank(rawText)){
			throw new SpiderException(SpiderExceptionCode.DataInvalideException);
		}
		JSONObject json = GsonUtils.fromJson(rawText,JSONObject.class);
		JSONArray jsonArray = json.getJSONObject("data").getJSONArray("detailList");
		if(jsonArray.size()==0 || jsonArray.isEmpty())
			throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        List<MovieCityModel> cityList = new ArrayList<>();
        String dataId = RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, page.getUrl().toString());
        if (StringUtils.isEmpty(dataId))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        for(int i = 0;i<jsonArray.size();i++){
        	JSONObject obj = (JSONObject) jsonArray.get(i);
            cityList.add(new MovieCityModel(dataId, Constants.SiteName.MAOYAN, obj.getString("name"),
            		obj.getString("boxInfo").replaceAll("[--]", "").equals("")?"":obj.getString("boxInfo"),
            		obj.getString("boxRate").replace("--", ""), 
            		obj.getString("showRate").replace("--", ""), 
            		obj.getString("avgShowView").replace("--", ""),
            		obj.getString("sumBoxInfo").replace("--", ""),
            		obj.getString("seatRate").replace("--", ""),
            		obj.getString("primeShowRate").replace("--", ""),
            		obj.getString("viewInfo").replace("--", ""),
                    obj.getString("showInfo").replace("--", "")));
        };

        page.putField(MovieCityModel.class.getName(), cityList);
    }

    /**
     * 影投数据
     * @param page
     */
    private void companyProcess(Page page) {
    	
    	String rawText = page.getRawText();
		if(null == rawText || StringUtils.isBlank(rawText)){
			throw new SpiderException(SpiderExceptionCode.DataInvalideException);
		}
		JSONObject json = GsonUtils.fromJson(rawText,JSONObject.class);
		JSONArray jsonArray = json.getJSONObject("data").getJSONArray("detailList");
		if(jsonArray.size()==0 || jsonArray.isEmpty())
			throw new SpiderException(SpiderExceptionCode.DataInvalideException);
		List<MovieShadowModel> shadowList = new ArrayList<MovieShadowModel>();
        String dataId = RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, page.getUrl().toString());
        if (StringUtils.isEmpty(dataId))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        for(int i = 0;i<jsonArray.size();i++){
        	JSONObject obj = (JSONObject) jsonArray.get(i);
        	MovieShadowModel movieShadowModel = new MovieShadowModel(
            		dataId,
            		Constants.SiteName.MAOYAN,
            		obj.getString("name"),
            		obj.getString("boxInfo").replaceAll("[--]", "").equals("")?"":obj.getString("boxInfo"),
            		obj.getString("boxRate").replace("--", ""), 
            		obj.getString("showRate").replace("--", ""), 
            		obj.getString("avgShowView").replace("--", ""),
            		obj.getString("sumBoxInfo").replace("--", ""),
            		obj.getString("seatRate").replace("--", ""),
            		obj.getString("primeShowRate").replace("--", ""),
            		obj.getString("viewInfo").replace("--", ""),
                    obj.getString("showInfo").replace("--", ""),
                    obj.getString("avgSeatView").replace("--", ""),
                    obj.getString("primeShow").replace("--", "")
                    );
        	shadowList.add(movieShadowModel);
        };

        page.putField(MovieShadowModel.class.getName(), shadowList);
		
	}
    
    private void peopleProcess(Page page)
    {
        Html html = PageUtils.spiderAudienceRating(page,cookies);
        List<String> divList = html.xpath("//div[@class='stackcolumn']").all();
        if (divList == null || divList.isEmpty())
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);

        List<String> ageNameList = html.xpath("//svg[@id='ageData']/g[@class='x']//text/text()")
                .all();
        List<String> ageValueList = html.xpath
                ("//svg[@id='ageData']/g[@class='bar-group']//text/text()").all();
        this.namesAndValuesValidate(ageNameList, ageValueList);

        String[][] sexData = this.getLineNamesAndValues(divList, "性别");
        String[][] eduData = this.getLineNamesAndValues(divList, "教育");

        List<String> areaNameList = html.xpath
                ("//section[@class='area-section']//div[@class='linebars-desc']/text()").all();
        List<String> areaValueList = html.xpath
                ("//section[@class='area-section']//div[@class='linebars-value']/text()").all();
        this.namesAndValuesValidate(areaNameList, areaValueList);

        String[][] occpData = this.getPieNamesAndValues(html, "occupData");
        String[][] labelData = this.getPieNamesAndValues(html, "interestData");

        String dataId = RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, page.getUrl().toString());
        page.putField(MoviePeopleModel.class.getName(), new MoviePeopleModel(dataId, Constants
                .SiteName.MAOYAN, sexData[0], sexData[1], ageNameList.toArray(new
                String[ageNameList.size()]), ageValueList.toArray(new String[ageValueList.size()
                ]), eduData[0], eduData[1], areaNameList.toArray(new String[areaNameList.size()])
                , areaValueList.toArray(new String[areaValueList.size()]), occpData[0],
                occpData[1], labelData[0], labelData[1]));
    }

    private String[][] getPieNamesAndValues(Html html, String tagId)
    {
        List<String> occpList = html.xpath("//svg[@id='" + tagId +
                "']/path[@class='pie-item']/@data-info").all();
        if (occpList == null || occpList.isEmpty())
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        List<String> occpNameList = occpList.stream().filter(data -> data.split(";").length > 1)
                .map(data -> data.split(";")[1]).collect(Collectors.toList());
        List<String> occpValueList = occpList.stream().filter(data -> data.split(";").length > 1)
                .map(data -> data.split(";")[0]).collect(Collectors.toList());
        this.namesAndValuesValidate(occpNameList, occpValueList);
        return new String[][]{occpNameList.toArray(new String[occpNameList.size()]),
                occpValueList.toArray(new String[occpValueList.size()])};
    }

    private String[][] getLineNamesAndValues(List<String> divList, String type)
    {
        String sexDiv = divList.stream().filter(divStr -> divStr.contains(type)).findFirst().get();
        if (StringUtils.isEmpty(sexDiv))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        Html sexHtml = new Html(sexDiv);
        List<String> nameList = sexHtml.xpath("//span[@class='stackcolumn-desc-text']/text()")
                .all();
        List<String> valueList = sexHtml.xpath("//i[@class='cs']/text()").all();
        this.namesAndValuesValidate(nameList, valueList);
        valueList = valueList.stream().map(value -> value + "%").collect(Collectors.toList());
        return new String[][]{nameList.toArray(new String[nameList.size()]), valueList.toArray(new
                String[valueList.size()])};
    }

    private void namesAndValuesValidate(List<String> nameList, List<String> valueList)
    {
        if (nameList == null || nameList.isEmpty())
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        if (valueList == null || valueList.isEmpty() || nameList.size() != valueList.size())
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
    }

    @Override
    protected String[] getStartRequestUrls()
    {
        return new String[]{String.format(LIST_URL, this.beginDate)};
    }

    @Override
    protected String getSpiderType()
    {
        return Constants.SpiderTaskType.MOVIE_BOXOFFICE;
    }

    @Override
    public String getPageDataType(Page page)
    {
        String url = page.getUrl().toString();
        if (url.contains("type=areaBox&method=change"))
            return Constants.PageDataType.TV_CITY;
        if (url.contains("type=shadowBox&method=change"))
            return Constants.PageDataType.MOVIE_COMPANY;
        if (url.contains("/wantindex"))
            return Constants.PageDataType.TV_AUDIENCE;
        if (url.contains("_v_=yes"))
            return Constants.PageDataType.TV_RECORD;
        if (url.contains("/marketing"))
            return Constants.PageDataType.MOVIE_MARKET;
        return Constants.PageDataType.SEARCH_LIST;
    }

}
